{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# transformers的简单入门"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "\n",
    "from transformers import BertTokenizer, BertModel, BertConfig\n",
    "# BertConfig模型配置的class\n",
    "# BertModel对应不同的Bert任务有不同的派生类，如BertForNextSentencePrediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at D:/review/bert-base-uncased were not used when initializing BertModel: ['cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "'''不需要手动下载模型文件'''\n",
    "# tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "# bert = BertModel.from_pretrained(\"bert-base-uncased\")\n",
    "'''需要手动下载模型文件'''\n",
    "\n",
    "# 读取模型对应的tokenizer\n",
    "tokenizer = BertTokenizer.from_pretrained('D:/review/bert-base-uncased')\n",
    "\n",
    "# 载入模型\n",
    "model = BertModel.from_pretrained(\"D:/review/bert-base-uncased\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 将text转化为token方法一"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[101, 2048, 4443, 2504, 3934, 2006, 2784, 4083, 102]\n"
     ]
    }
   ],
   "source": [
    "input = \"Two entry level projects on deep learning\"\n",
    "\n",
    "# 通过tokenizer把文本变成token_ids\n",
    "input_ids = tokenizer.encode(input, add_special_tokens=True)\n",
    "print(input_ids)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 将text转化为token方法二"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[101, 2048, 4443, 2504, 3934, 2006, 2784, 4083, 102]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 将模型拆分成token词汇中可用的token\n",
    "tokens = tokenizer.tokenize(input)\n",
    "tokens = [\"[CLS]\"] + tokens + [\"[SEP]\"]\n",
    "input_ids_2 = tokenizer.convert_tokens_to_ids(tokens)\n",
    "input_ids_2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 上面两种方法生成的ids相等"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert [i for i in input_ids] == [j for j in input_ids_2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[-0.4271, -0.1594, -0.3350,  ..., -0.0228,  0.1340,  0.3698],\n",
      "         [-0.4425, -0.4270, -0.4322,  ...,  0.2256,  0.1559,  0.1738],\n",
      "         [-0.7596,  0.3395,  0.8543,  ...,  0.4581,  0.1042, -0.1535],\n",
      "         ...,\n",
      "         [-0.6593, -0.4336,  0.2952,  ..., -0.0948, -0.4339,  0.4047],\n",
      "         [-1.1158, -0.7114, -0.6332,  ..., -0.0349,  0.0247, -0.0130],\n",
      "         [ 0.6916, -0.1245, -0.4680,  ...,  0.3236, -0.7338, -0.1450]]])\n",
      "torch.Size([1, 9, 768])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "input_ids = torch.tensor(input_ids)\n",
    "\n",
    "x = input_ids.unsqueeze(0)\n",
    "\n",
    "# 输出最后一个隐藏的结果\n",
    "with torch.no_grad():# 能进行梯度计算的上下文管理器\n",
    "    result = model(x)[0]\n",
    "print(result)\n",
    "print(result.shape)# 对文本的每一个token生成一个768维的张量"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [101, 2048, 4443, 2504, 3934, 2006, 2784, 4083, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1]}"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input = \"Two entry level projects on deep learning\"\n",
    "tokenizer(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[101, 2048, 4443, 2504, 3934, 2006, 2784, 4083, 102]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer(input)[\"input_ids\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[CLS] two entry level projects on deep learning [SEP]'"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_sequence = tokenizer(input)[\"input_ids\"]\n",
    "tokenizer.decode(encoded_sequence)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### tokenizer中的padding属性\n",
    "+ padding=True  按最大维度拼接\n",
    "+ padding=False  直接拼接"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'input_ids': [101, 2023, 2003, 1037, 2460, 5537, 1012, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1]}\n",
      "{'input_ids': [101, 2023, 2003, 1037, 2738, 2146, 5537, 1012, 2009, 2003, 2012, 2560, 2936, 2084, 1996, 5537, 1037, 1012, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}\n",
      "{'input_ids': [[101, 2023, 2003, 1037, 2460, 5537, 1012, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [101, 2023, 2003, 1037, 2738, 2146, 5537, 1012, 2009, 2003, 2012, 2560, 2936, 2084, 1996, 5537, 1037, 1012, 102]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}\n",
      "====================万能的分隔符====================\n",
      "{'input_ids': [[101, 2023, 2003, 1037, 2460, 5537, 1012, 102], [101, 2023, 2003, 1037, 2738, 2146, 5537, 1012, 2009, 2003, 2012, 2560, 2936, 2084, 1996, 5537, 1037, 1012, 102]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}\n"
     ]
    }
   ],
   "source": [
    "sequence_a = \"This is a short sequence.\"\n",
    "sequence_b = \"This is a rather long sequence.It is at least longer than the sequence A.\"\n",
    "\n",
    "encoded_sequence_a = tokenizer(sequence_a)\n",
    "encoded_sequence_b = tokenizer(sequence_b)\n",
    "\n",
    "print(encoded_sequence_a)\n",
    "print(encoded_sequence_b)\n",
    "\n",
    "print(tokenizer([sequence_a, sequence_b], padding=True))\n",
    "print(20*\"=\" + \"万能的分隔符\"+ 20*\"=\")\n",
    "print(tokenizer([sequence_a, sequence_b]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
